import scrapy


class QidianSpider(scrapy.Spider):
    name = 'qidian'
    allowed_domains = ['www.qidian.com']

    start_urls = ['https://www.qidian.com/all/', 'https://www.qidian.com/mm/all/']

    # def start_requests(self):
    #     size = 0
    #     while (size < 5):
    #         size = size + 1
    #         yield scrapy.Request(f'http://www.qidian.com/all/page{size}', self.parse)

    def parse(self, response):
        # print(response.text)
        # names = response.xpath('//ul/li/div[2]/h2/a/text()').extract()
        # authors = response.xpath('//*[@id="book-img-text"]/ul/li/div[2]/p[1]/a[1]/text()').extract()
        # book = []
        # for name, author in zip(names, authors):
        #     book.append({'name': name, 'author': author})
        # print(book)
        # return book

        for href in response.xpath('//*[@id="page-container"]/div/ul/li/a/@href').getall():
            # print(href)
            yield scrapy.Request('http:' + href, self.parse_item)

        # return book
        # pass

    def parse_item(self, response):
        names = response.xpath('//ul/li/div[2]/h2/a/text()').extract()
        authors = response.xpath('//*[@id="book-img-text"]/ul/li/div[2]/p[1]/a[1]/text()').extract()
        book = []
        for name, author in zip(names, authors):
            book.append({'name': name, 'author': author})
        return book
